# -*- coding: utf-8 -*-  
'''
初始化字典

@author: luoyi
Created on 2021年6月29日
'''
import utils.conf as conf
import utils.dictionaries as dicts

import data.dataset_baidu as ds_baidu


#    初始化字典
dicts.load_dict_from_chchars(chchars_path=conf.TEXT.get_dictionaries_chchars_path())
dicts.auto_append_on()
print('当前字典容量:', dicts.dict_size())

#    用训练集追加字典
ds_iterator = ds_baidu.dataset_iterator(file_path=conf.DATASET_BAIDU.get_train_data_path())
for s in ds_iterator:
    dicts.word2idx_slist(s.text)
    pass

print('补全后字典容量:', dicts.dict_size())
#    保存字典
dicts.save_dict_to_pkl()